# -*- coding: utf-8 -*-

from goose import Goose
from goose.text import StopWordsChinese

class crawl(object):
    _instance = None

    def __init__(self):
        self._g = Goose({'stopwords_class': StopWordsChinese})

    @classmethod
    def has_instance(cls):
        return cls._instance is not None

    @classmethod
    def _create_instance(cls):
        if not cls.has_instance():
            cls._instance = cls()

    @classmethod
    def get_instance(cls):
        if not cls.has_instance():
            cls._create_instance()
        return cls._instance

    def fetch(self, url):
        return self._g.extract(url=url)

def get_crawl():
    return crawl.get_instance()

class extract(object):

    @classmethod
    def get_all_links(self, dom):
        domain = dom.domain
        # lxml.etree._ElementStringResult lists
        _urls =  dom.doc.xpath('//a/@href')
        urls = []
        for _url in _urls:
            try:
                _url = str(_url)
            except UnicodeEncodeError:
                continue
            # check empty str('')
            if not _url:
                continue
            if not _url.startswith('http'):
                _url = 'http://' + domain + '/' + _url
            urls.append(_url)
        return urls

    @classmethod
    def get_main_content(self, dom):
        content = {
            'content': dom.cleaned_text if dom.cleaned_text else '',
            'images':  [dom.top_image.src if dom.top_image else ''],
            'title': dom.title if dom.title else ''
        }
        return content


def get_extract():
    return extract()


